package module::stylebop_au::default;
use Encode;
use vars qw($VERSION @ISA @EXPORT);
$VERSION = 1.00;
@ISA     = qw(Module::Module);
@EXPORT  = ();
use strict;

# ----------------------------------------------------
# constructor
# 3/7/2013 Louise create
# ----------------------------------------------------
sub new {
	my $class = shift;
	my %parm  = @_;
	my $this  = {};
	bless $this, $class;
	return $this;
}

# ----------------------------------------------------
# Get all product's infos from affiliatefeed then write 
# them into temp xml
# Modify Log
# 3/7/2013 Louise create
# ----------------------------------------------------
sub filter {

	#A200000 Filter retailer website starts
	Log::Write::setLog('A200000');
	my $this     = shift;
	my $writexml = new Feed::XML();
	my $data     = $main::filedata;
	my $hostURL  = $this->{'config'}->{'host_url'};
	my $catalog  = { 'Product' => [] };
	my $charset  = 'iso-8859-1';
	my $splitString = '<Split>';
	
	# get Women categoryUrls and Products
	my @allProducts = &getCategoryUrls($hostURL, $charset, $splitString);
	# write tree date into XML file
	if (@allProducts) {
		$catalog->{'Product'} = \@allProducts;
		$writexml->addXML( $catalog, $this->{'config'}->{'data_filter_file'} );
		# A200002the total count of detail page urls
		Log::Write::setLog( 'A200002', { 'COUNTTOTAL' => scalar(@allProducts) } );
	}
	else {
		# E200000filter failed
		Log::Write::setLog('E200000');
		return;
	}

	# A299999Filter retailer website end.
	Log::Write::setLog('A299999');
			
}
	
# ----------------------------------------------------
# Parse all products detail page and write the infos.
# into final XML feed.
# Modify Log
# 3/7/2013 Louise create
# ----------------------------------------------------
sub parser {
	
	# A300000 Parser detail page starts
	Log::Write::setLog('A300000');
	
	my $this = shift;
	my $writeXml = new Feed::XML();
	my $hostURL  = $this->{'config'}->{'host_url'};
	my $catalog  = { 'Product' => [] };
	my $charset  = 'iso-8859-1';
	my $splitString = '<Split>';
	
	# new parser object
	my $parserObj = module::stylebop_au::parser->new;
	
	# read xml create by filter to get all products infos(url and category)
	my @xmlInfo = $writeXml->readXML( $this->{'config'}->{'data_filter_file'} );

	my $count = 0;
	my $total = 0;
	
	for my $oneTemp (@xmlInfo) {
	 	my %oneTempInfoHash = %{$oneTemp};
		my $url	= ( @{ $oneTempInfoHash{'ProductURL'} } )[0];
		my %productInfoHash;
		eval {
			# get infos from product detail page and save it in hash variable
			%productInfoHash = $parserObj->parseAndgetInfo($url, $charset, $hostURL);
		};
		if ($@) {
			# E300003 : Failed to extract information from XXX page.
			Log::Write::setLog( "E300003", { 'URL' => $url } );
			next;
		}
		
		if ( $oneTempInfoHash{'Name'} ) {
	    	my $product = { 'Retailer'    => $this->{ 'config' }->{ 'retailer_name' },
							'ProductId'   => $productInfoHash{ 'ProductId' },
							'Name'        => $oneTempInfoHash{ 'Name' },
							'Brand'       => $oneTempInfoHash{ 'Brand' },
							'Price'       => ($oneTempInfoHash{ 'Price' })?$oneTempInfoHash{ 'Price' }:$productInfoHash{ 'Price' },
							'SalePrice'   => ($oneTempInfoHash{ 'SalePrice' })?$oneTempInfoHash{ 'SalePrice' }:$productInfoHash{ 'SalePrice' },
							'Description' => $productInfoHash{ 'Description' },
							'ProductURL'  => $oneTempInfoHash{ 'ProductURL' },
							'ImageURL'    => $productInfoHash{ 'ImageURL' },
							'Category'    => $oneTempInfoHash{ 'Category' },
							'Size'        => $productInfoHash{ 'Size' },
							'Time'        => $productInfoHash{ 'Time' },
	                     };
	       $catalog->{ 'Product' } = [ @{ $catalog->{ 'Product' } }, $product ];
	    }

	    $count++;
	    $total++;
	    
	    # write data into XML per 100(configurable) records 
	    if ($count == $this->{'config'}->{'write_frequency'}) {
			$writeXml->addXML( $catalog, $this->{'config'}->{'data_feed_file'} );
			$catalog     = { 'Product' => [] };
			# A300001 : Record XXX prodcuts have been appended to feed file.
			Log::Write::setLog( "A300001", {'COUNT'=>$count});
			$count = 0;
		}
	}
	$writeXml->addXML( $catalog, $this->{'config'}->{'data_feed_file'} );
    
    # A300001 : Record XXX prodcuts have been appended to feed file.
	Log::Write::setLog( "A300001", { 'COUNT' => $count } );

	if ($total) {
	    # A300003 : Record XXX prodcuts have been written into feed file totally.
		Log::Write::setLog( 'A300003', { 'COUNT' => $total } );
		# A399999 : Parser detail page end.
		Log::Write::setLog('A399999');
	}
	else {
		# E300000 : Parser failed.
		Log::Write::setLog('E300000');
	}
	return;
}

# ----------------------------------------------------
# get all category urls
# Modify Log
# 3/7/2013 Louise create
# ----------------------------------------------------
sub getCategoryUrls {
	
	my $url     = shift;
	my $charset = shift;
	my $splitString = shift;
	my @allProducts = ();
	my @catArray = ();
	
	# get page failed
	my $content = &getGenderPage( $url, $charset );
	if (!$content) {
		# E200001 Failed to get XX entrance information.
		Log::Write::setLog( 'E200001', { 'URL' => $url } );
		return;
	}else{
		$content =~ s/<!--.*?-->//isg;
	}
	# Get the gender url and push into the  genderURL Array.
	my $genderWName = 'Women';
	my $genderMName = 'Men';
	my @genderUrlArray = ();
	if ($content =~ /genderrow.*?<a\s*href=["']([^"]*)["']>.*?<a\s*href=["']([^"]*)["']/is){
		my $genderWUrl = $1;
		my $genderMUrl = $2;
		push @genderUrlArray, $genderMUrl.$splitString.$genderMName.$splitString.'_men';
		push @genderUrlArray, $genderWUrl.$splitString.$genderWName.$splitString;
	}
	# Check the genderUrlArray,if it is null,return.
	if (!@genderUrlArray){
		# E200001 Failed to get XX entrance information.
		Log::Write::setLog( 'E200001', { 'URL' => $url } );		
		return;
	}
	
	# Get the category url by each gender.
	my @product1 = ();
	my @product2 = ();
	my @product3 = ();
	my @product4 = ();
	my @product5 = ();
	my @product6 = ();
	foreach my $genderUrl (@genderUrlArray){
		my ($genderUrlTemp, $genderNameTemp, $mainItemTemp) = split(/\Q$splitString\E/, $genderUrl);
		my $content = &getGenderPage( $genderUrlTemp, $charset, $mainItemTemp);
		if ($content =~ /<div\s*id="menurow">(.*?)contentcontainer/is){
			my $navText = $1;
			my $categoryPrefix = 'http://www.stylebop.com/search.php?state=1&';
			while($navText =~ /<div\s*class=["']mainmenuitem$mainItemTemp["']>.*?<a\s*class=["']link[_-]mainmenu["']\s*href="([^"]*)">.*?alt="([^"]*)"(.*?)<\/div>\s*<\/div>\s*<\/li>/isg){
				my $cat2Url = $1;
				my $cat2Name = $2;
				my $navContent = $3;
				$cat2Name =~ s/&amp;/&/isg;
	        	$cat2Name =~ s/^\s+|\s+$//isg;
	        	if ($cat2Name =~ /Designers|Shop\sBy\sLook|Magazine/is){
	            	next;
	        	}
	        	# special for category New Arrivals
	        	if ($cat2Name =~ /New\s*Arrivals/is){
        			my $tempUrl2 = $cat2Url;
	        		my $tempText = &getGenderPage($tempUrl2, $charset, $mainItemTemp);

	        		if ($tempText =~ /var\s*defaultQueryString\s*=\s*["']\?*([^"']*)["']/is){
	        			my $cat2TempUlr = $1;
	        			$tempUrl2 = $categoryPrefix.$cat2TempUlr;
	        			push @catArray, $tempUrl2.$splitString.$genderNameTemp.$splitString.$cat2Name;
	        			Log::Write::setLog( 'A200001', { 'url' => $tempUrl2, 'category' => $genderNameTemp.$splitString.$cat2Name } );
	        			my @productTemp1 = &getFilterProduct( $charset, [$tempUrl2.$splitString.$genderNameTemp.$splitString.$cat2Name], $splitString, $url, $mainItemTemp);
	        			push @product1, @productTemp1;
	        		}
	        		
	        		my $temp3Text = &getGenderPage($cat2Url, $charset, $mainItemTemp);
	        		while ($temp3Text =~ /MenuLeft2["']\s*href=["']([^"']*)["'][^>]*>[^>]*>([^<]*)/isg){
	        			my $Temp3Ulr = $1;
	        			my $cat3Name = $2;
	        			$cat3Name =~ s/&amp;/&/isg;
	        			my $text3 = &getGenderPage($Temp3Ulr, $charset, $mainItemTemp);
	        			if ($text3 =~ /var\s*defaultQueryString\s*=\s*["']\?*([^"']*)["']/is){
	        				my $cat3TempUlr = $1;
	        				my $cat3Url = $categoryPrefix.$cat3TempUlr;
	        				push @catArray, $cat3Url.$splitString.$genderNameTemp.$splitString.$cat2Name.$splitString.$cat3Name;
	        				Log::Write::setLog( 'A200001', { 'url' => $cat3Url, 'category' => $genderNameTemp.$splitString.$cat2Name.$splitString.$cat3Name } );
	        				my @productTemp2 = &getFilterProduct( $charset, [$cat3Url.$splitString.$genderNameTemp.$splitString.$cat2Name.$splitString.$cat3Name], $splitString, $url, $mainItemTemp);
	        				push @product2, @productTemp2;
	        			}
	        		}
	        	}
	        	
	        	# crawl category3
	        	while ($navContent =~ /<a\s*class=["']link_submenu["']\s*href="([^"]*)"[^>]*>\s*(.*?)<\/a>/isg) {
	        		my $cat3Url = $1;
	        		my $cat3Name = $2;
	        		$cat3Name =~ s/&amp;/&/isg;
	        		if ($cat3Url eq $cat2Url) {
	        			next;
	        		}
	        		my $lastContent = '';
	        		if ($cat3Name =~ /Shop\s*by\s*Designer|All/s){
	        			next;
	        		}
	        		$lastContent = &getGenderPage($cat3Url, $charset, $mainItemTemp);
	        		if ($lastContent =~ /var\s*defaultQueryString\s*=\s*["']\?*([^"']*)["']/is){
						$cat3Url = $categoryPrefix.$1;
						push @catArray, $cat3Url.$splitString.$genderNameTemp.$splitString.$cat2Name.$splitString.$cat3Name;
						Log::Write::setLog( 'A200001', { 'url' => $cat3Url, 'category' => $genderNameTemp.$splitString.$cat2Name.$splitString.$cat3Name} );
						my @productTemp3 = &getFilterProduct( $charset, [$cat3Url.$splitString.$genderNameTemp.$splitString.$cat2Name.$splitString.$cat3Name], $splitString, $url, $mainItemTemp);
						push @product3, @productTemp3;
	        		}else{
						Log::Write::setLog( 'E200001', { 'url' => $cat3Url } );
					}
	        		# crawl category4
	        		while ($lastContent =~ /<a\s*class=["']MenuLeft2New["'][^>]*\(["']([^'"]*)[^>]*><div>([^<]*)/isg){
	        			my $cat4TempUlr = $1;
	        			my $cat4Name = $2;
	        			$cat4Name =~ s/&amp;/&/isg;
	        			my $cat4Url = $categoryPrefix.$cat4TempUlr;
	        			push @catArray, $cat4Url.$splitString.$genderNameTemp.$splitString.$cat2Name.$splitString.$cat3Name.$splitString.$cat4Name;
	        			Log::Write::setLog( 'A200001', { 'url' => $cat4Url, 'category' => $genderNameTemp.$splitString.$cat2Name.$splitString.$cat3Name.$splitString.$cat4Name } );
	        			my @productTemp4 = &getFilterProduct( $charset, [$cat4Url.$splitString.$genderNameTemp.$splitString.$cat2Name.$splitString.$cat3Name.$splitString.$cat4Name], $splitString, $url, $mainItemTemp);
	        			push @product4, @productTemp4;
	        		}
	        		# special for sale category
	        		if ($cat2Name =~ /sale/is){
		        		while ($lastContent =~ /<a\s*class=["']MenuLeft2["']\s*href=["']([^"']*)["']><div\s*class=["']leftmenuitem["']>([^<]*)/isg){
		        			my $cat4TempUlr = $1;
		        			my $cat4Name = $2;
		        			$cat4Name =~ s/&amp;/&/isg;
		        			if ($cat4Name =~ /All/s){
			        			next;
			        		}
			        		if ($cat3Name =~ /Last\sPiece/is && $cat4Name =~ /Lingerie/is){
			        			next;
			        		}
		        			my $lastContent5 = &getGenderPage($cat4TempUlr, $charset, $mainItemTemp);
		        			my $cat4Url = '';
		        			if ($lastContent5 =~ /var\s*defaultQueryString\s*=\s*["']\?*([^"']*)["']/is){
								$cat4Url = $categoryPrefix.$1;
							} else {
								$cat4Url = $cat4TempUlr;
		        			}
							push @catArray, $cat4Url.$splitString.$genderNameTemp.$splitString.$cat2Name.$splitString.$cat3Name.$splitString.$cat4Name;
							Log::Write::setLog( 'A200001', { 'url' => $cat4Url, 'category' => $genderNameTemp.$splitString.$cat2Name.$splitString.$cat3Name.$splitString.$cat4Name} );
		        			my @productTemp5 = &getFilterProduct( $charset, [$cat4Url.$splitString.$genderNameTemp.$splitString.$cat2Name.$splitString.$cat3Name.$splitString.$cat4Name], $splitString, $url, $mainItemTemp);
		        			push @product5, @productTemp5;
		        			# crawl category5
		        			while ($lastContent5 =~ /<a\s*class=["']MenuLeft2New["'][^>]*\(["']([^'"]*)[^>]*><div>([^<]*)/isg){
		        				my $cat5TempUlr = $1;
		        				my $cat5Name = $2;
		        				$cat5Name =~ s/&amp;/&/isg;
		        				my $cat5Url = $categoryPrefix.$cat5TempUlr;
		        				push @catArray, $cat5Url.$splitString.$genderNameTemp.$splitString.$cat2Name.$splitString.$cat3Name.$splitString.$cat4Name.$splitString.$cat5Name;
		        				Log::Write::setLog( 'A200001', { 'url' => $cat5Url, 'category' => $genderNameTemp.$splitString.$cat2Name.$splitString.$cat3Name.$splitString.$cat4Name.$splitString.$cat5Name } );
		        				my @productTemp6 = &getFilterProduct( $charset, [$cat5Url.$splitString.$genderNameTemp.$splitString.$cat2Name.$splitString.$cat3Name.$splitString.$cat4Name.$splitString.$cat5Name], $splitString, $url, $mainItemTemp);
		        				push @product6, @productTemp6;
		        			}
		        		}
	        		}
	        	}
			}
		}
	}
	push @allProducts, @product1;
	push @allProducts, @product2;
	push @allProducts, @product3;
	push @allProducts, @product4;
	push @allProducts, @product5;
	push @allProducts, @product6;
	return @allProducts;
}

# ----------------------------------------------------
# get all entry urls
# 3/7/2013 Louise create
# ----------------------------------------------------
sub getFilterProduct {

	my $charset      = shift;
	my $entryUrls    = shift;
	my $splitString  = shift;
	my $hostURL = shift;
	my $genderName = shift;
	my $data    = $main::filedata;
	my @ProductsArray;

	for my $entryUrlRef (@{$entryUrls}) {
		my ($url, @catArray) = split($splitString, $entryUrlRef);
		
		my $pageCount = 1;
		for (1..30){
			my $urlPage = $url.'&page='.$pageCount;
			my $newContent = &getGenderPage($urlPage, $charset, $genderName);

			# get page failed
			if (!$newContent) {
				# E200001 Failed to get XX entrance information.
				Log::Write::setLog( 'E200001', { 'URL2' => $urlPage } );
				next;
			}else{
				$newContent =~ s/<!--.*?-->//isg;
			}
			
			if ($newContent !~ /productitem/is){
				last;
			}
			# get the rest of pages
			my @restUrlArray = ();
			my $brand = '';
			my $productUrl = '';
			my $name = '';
			my $price = '';
			my $salePrice = '';
			my $productId = '';
			my $tempUrl = '';
			my $char1 = decode('utf-8', 'É');
				
			if ($newContent =~ /products\s*[_-]*container(.*?)totop/is){
				my $linkText = $1;
				my @tempArray = ();
				while ($linkText =~ /product[-_]*item(.*?)search[_-]add.*?itemCondition[^>]*>([^<]*)/isg){
					my $oneProductText = $1;
					my $stock = $2;
					if($stock =~ /sold\s*out/is){
						next;
					}
					if ($oneProductText =~ /article.*?href=["']([^"']*)["'][^>]*>([^<]*)<\/a>/is){
						$productUrl = $1;
						$name = $2;
						$productUrl =~ s/&amp;/&/isg;
						$name =~ s/&amp;/&/isg;
						if ($productUrl =~ /id=(\d+)$/is){
							$productId = $1;
						}elsif($productUrl =~ /id=(\d+)\&special=sale/is){
							$productId = $1;
						}
					}
					# failed get ProductId
					if(!$productId){
						Log::Write::setLog( "R200008", { 'URL' => $productUrl, 'POINT' => 'ProductId' } );
					}
					# failed get ProductURL
					if(!$productUrl){
						Log::Write::setLog( "E200008", { 'URL' => $productUrl, 'POINT' => 'ProductURL' } );
						next;
					}
					if ($oneProductText =~ /brand.*?>([^<]*)<\/a>/is){
						$brand = $1;
						$brand =~ s/&amp;/&/isg;
						$brand =~ s/&Eacute;/$char1/isg;
					}
					if (!$brand && $oneProductText =~ /["']$name\s*by\s*([^"']*)["']/is){
						$brand = $1;
						$brand =~ s/&amp;/&/isg;
						$brand =~ s/&Eacute;/$char1/isg;
					}
					#failed get Brand
					if(!$brand){
						Log::Write::setLog( "R200008", { 'URL' => $productUrl, 'POINT' => 'Brand' } );
					}
					#failed get ProductName
					if(!$name){
						Log::Write::setLog( "R200008", { 'URL' => $productUrl, 'POINT' => 'Name' } );
					}
					if ($oneProductText =~ /search[_-]price(.*?)<\/div>/is){
						my $priceText = $1;
						if ($priceText =~ />([^<]*)<\/span>.*?sale[_-]*price>([^<]*)<\/span>/is){
							$price = $1;
							$salePrice = $2;
							$salePrice =~ s/\(.*?\)//isg;
							$price =~ s/\(.*?\)//isg;
							$price =~ s/^\s+|\s+$//isg;
							$salePrice =~ s/^\s+|\s+$//isg;
						}elsif ($priceText =~ />([^<]*)<\/span>/is ){
							$price = $1;
							$price =~ s/\(.*?\)//isg;
							$price =~ s/^\s+|\s+$//isg;
						}
					}
					#failed get Price and SalePrice
					if(!$price){
						Log::Write::setLog( "R200008", { 'URL' => $productUrl, 'POINT' => 'Price' } );
					}
					
					$name = $brand.' '.$name;
					
					my $product = {
									'ProductId'  => $productId,
									'ProductURL' => $productUrl,
									'Name' => $name,
									'Brand' => $brand,
									'Price' => $price,
									'SalePrice' => $salePrice,
									'Category'    => [{'Part' => \@catArray}],
								};
					push @tempArray, $product;
				}
				Log::Write::setLog( 'A200001', { 'Count' => scalar(@tempArray), 'url'=>$urlPage} );
				push @ProductsArray, @tempArray;
			}
			$pageCount++;
		}
	}
	# A200003 The total count of the entryUrl obtained
	Log::Write::setLog( 'A200003', { 'COUNT' => scalar(@ProductsArray) } );
	return @ProductsArray;
}

sub getGenderPage {
	my $urlPage = shift;
	my $charset = shift;
	my $genderName = shift || '';
	my $data = $main::filedata;
	my $useragent = LWP::UserAgent->new();
	my @header = (
	        "Host"=>"www.stylebop.com",
	        "User-Agent" => "Mozilla/5.0 (Windows NT 5.1; rv:19.0) Gecko/20100101 Firefox/19.0",
	        "Accept" => "text/html, */*; q=0.01",
	        "Accept-Language" => "zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3",
	        "Connection" => "keep-alive",
	        "X-Requested-With" => "XMLHttpRequest",
	        "Cookie"=>"PHPSESSID=5b3868cdfa63ab8c5dd23587e8b745b8; __utma=66782444.1849209937.1363327741.1363327741.1363327741.1; __utmb=66782444.12.10.1363327741; __utmc=66782444; __utmz=66782444.1363327741.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); ia_bncl_2333636313236323131303=0.03967958994108278%201363327742%201363327910%204%20168; ia_c4dc_2333636313236323131303=1; ia_u4pc_2333636313236323131303=1; emos_jcsid=AAABPWyqdGZPUU6o8_htmDXnv9QurMuG:4:AAABPWytCqd6MKIKHrU2CuTM5Jhx_2HP:1363327912615; emos_jcvid=AAABPWyqdGZPUU6o8_htmDXnv9QurMuG:1:AAABPWyqdGZPUU6o8_htmDXnv9QurMuG:1363327743077:0:false:10"
	    );
	if ($genderName eq '_men') {
		@header = (
	        "Host"=>"www.stylebop.com",
	        "User-Agent" => "Mozilla/5.0 (Windows NT 5.1; rv:19.0) Gecko/20100101 Firefox/19.0",
	        "Accept" => "text/html, */*; q=0.01",
	        "Accept-Language" => "zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3",
	        "Connection" => "keep-alive",
	        "X-Requested-With" => "XMLHttpRequest",
	        "Cookie"=>"	PHPSESSID=b74cbe3a29d674702ad1aa828f56d22a; __utma=66782444.1645206282.1364027279.1364027279.1364027279.1; __utmb=66782444.21.10.1364027279; __utmc=66782444; __utmz=66782444.1364027279.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); ia_bncl_2333636313236323131303=0.11497609902345007%201364027281%201364027790%207%20509; ia_c4dc_2333636313236323131303=1; ia_u4pc_2333636313236323131303=1; emos_jcsid=AAABPZZck8xgWWlzaPH*w2*W*6AwNkPE:7:AAABPZZkZF0cch16F2DdPzS5rz_F3x2H:1364027794525; emos_jcvid=AAABPZZck8xgWWlzaPH*w2*W*6AwNkPE:1:AAABPZZck8xgWWlzaPH*w2*W*6AwNkPE:1364027282379:0:false:10"
	    );
	}
#	my $newContent = $data->getPage($urlPage);
#	my $cookie_jar = HTTP::Cookies->new;
#	$useragent->cookie_jar($cookie_jar);
	my $request = HTTP::Request->new(GET=>$urlPage);
	$request->header(@header);
	my $response = $useragent->request($request);
	my $newContent = $response->content();
	$newContent = decode($charset, $newContent);
	return $newContent;
}

1;